from __future__ import division

inputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTempleteV3_phase0_1_2.txt.output.20121023"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data5/team/weijiang/the_new_trip_of_feature_generation/wholeTrainingFileTempleteV3_phase0_1_2.txt.output.20121023_fixed"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    
    # check1: check whether each line has the same number of elements
    if len(lineElements) != 81:
        print "error, the number of elements is:",len(lineElements)
        exit(1)
    
    # fix1: currently, the url domain has many values, and normalize to fit the xls file
    if lineElements[14] == "com" or lineElements[14] == "edu" or lineElements[14] == "gov" or lineElements[14] == "org" or lineElements[14] == "net":
        pass
    else:
        lineElements[14] = "other"
    
    
    # fix2 for the sum/avg/min/max term_rel_freq_in_doc (by Wei to fit Maria)
    # print lineElements[16],lineElements[-9],lineElements[-8],lineElements[-7],lineElements[-6]
    # +24 (since we now have the positional features)
    if float(lineElements[16]) != 0:
        lineElements[-41] = str( float(lineElements[-17]) / float(lineElements[16]) )
        lineElements[-40] = str( float(lineElements[-16]) / float(lineElements[16]) )
        lineElements[-39] = str( float(lineElements[-15]) / float(lineElements[16]) )
        lineElements[-38] = str( float(lineElements[-14]) / float(lineElements[16]) )
    else:
        lineElements[-41] = "0"
        lineElements[-40] = "0"
        lineElements[-39] = "0"
        lineElements[-38] = "0"
    
       
    # fix3 terms_in_header currently is 0 always, but it really deserve the same value of how the field terms_in_title does.
    lineElements[26] = lineElements[27]
    
    # fix4 frac_terms_in_header currently is 0 always, but it really deserve the same value of how the field frac_terms_in_title does.
    lineElements[31] = lineElements[32]
    
    # fix5: why the avg_term_BM25 is always 0, and this problem has been fixed from the original point also.
    lineElements[45] = str( float(lineElements[44]) / int(lineElements[4]))
    
    
    # fix6: deal with the frequency features Maria especially mentioend in her email. (2012/10/23)
    # from the index: 36
    #
    # print "lineElements[36-39]:",lineElements[36],lineElements[37],lineElements[38],lineElements[39]
    sum_term_freq_in_doc_old = int(lineElements[36])
    min_term_freq_in_doc_old = int(lineElements[37])
    max_term_freq_in_doc_old = int(lineElements[38])
    queryLength = int(lineElements[4])
    docSize = int(lineElements[16])
    
    if sum_term_freq_in_doc_old == min_term_freq_in_doc_old and min_term_freq_in_doc_old == max_term_freq_in_doc_old and queryLength != 0:
        # in this situation, some query terms must have been missing, so I should set the min_term_freq_in_doc_old to 0 instead
        lineElements[37] = "0"
    
    # fix7: recompute the following 
    # sum_term_rel_freq_in_doc
    # min_term_rel_freq_in_doc
    # max_term_rel_freq_in_doc
    # avg_term_rel_freq_in_doc  
    # print "lineElements[40-43]:",lineElements[40],lineElements[41],lineElements[42],lineElements[43]
    if docSize != 0:
        lineElements[40] = str( int( lineElements[36] ) / docSize )
        lineElements[41] = str( int( lineElements[37] ) / docSize )
        lineElements[42] = str( int( lineElements[38] ) / docSize )
        lineElements[43] = str( float( lineElements[39] ) / docSize )
    else:
        # Just let the original computation score show up
        pass
    
    
    
    
    
    # there is another fixed for the the Maria's email and that will be done.
    # I love you so much.
    
    
    
    #output logic
    newOutputLine = ""
    for element in lineElements:
        newOutputLine += element + " "
    
    outputFileHandler.write(newOutputLine + "\n")
    

inputFileHandler.close()
outputFileHandler.close()
print "all fixed done"
    
    
    
